{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.2"
    },
    "colab": {
      "name": "Electra.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "01d8ad27e05a4175b9454d7ebc2363ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4894baf418d441b89f9694c33c4a8221",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_572c7ef1afd547e8a5dfe5ee26162200",
              "IPY_MODEL_9d2fe16e49164d32b42823907c980af0"
            ]
          }
        },
        "4894baf418d441b89f9694c33c4a8221": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "572c7ef1afd547e8a5dfe5ee26162200": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_76dec96200924d8393a421001a7a54bd",
            "_dom_classes": [],
            "description": "models/electra/generator/pytorch_model.bin: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 260982246,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 260982246,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f6bd2538ae1b42c9aed28b231d3cee9e"
          }
        },
        "9d2fe16e49164d32b42823907c980af0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6af790cfb9c446679fe60fd8a98078db",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 249M/249M [02:24&lt;00:00, 1.80MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0d24fa9e2c2e4619ad9e78bc7b054c7f"
          }
        },
        "76dec96200924d8393a421001a7a54bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f6bd2538ae1b42c9aed28b231d3cee9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6af790cfb9c446679fe60fd8a98078db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0d24fa9e2c2e4619ad9e78bc7b054c7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16277ad20d5349a8a737a54acd2ee506": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_803da87847ce429d9b06b3905ccf24be",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7277eafb8b5f47f39f1709be9ebfcce3",
              "IPY_MODEL_2bd8bc9cf0c64c419781a87d97842bff"
            ]
          }
        },
        "803da87847ce429d9b06b3905ccf24be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7277eafb8b5f47f39f1709be9ebfcce3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_927fae41ae594686b6504d1c4ec84924",
            "_dom_classes": [],
            "description": "models/electra/discriminator/pytorch_model.bin: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1679849201,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1679849201,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1c7a033d87f94f6eb5aceb5e49abe8c5"
          }
        },
        "2bd8bc9cf0c64c419781a87d97842bff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_712c6b0ebfed4b189d3415eee814ce2e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.56G/1.56G [02:06&lt;00:00, 13.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c90762cb282b40eea7c3d084c8841b0e"
          }
        },
        "927fae41ae594686b6504d1c4ec84924": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1c7a033d87f94f6eb5aceb5e49abe8c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "712c6b0ebfed4b189d3415eee814ce2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c90762cb282b40eea7c3d084c8841b0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "14e2f1e4f16845a2bf29b4bf3cdfc008": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_40069cf306434810a9b5c306cc250b4d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_67561c38a4db4ddd8b1bbf2e583efac0",
              "IPY_MODEL_445908c8697442ee9c64839c34bfaeed"
            ]
          }
        },
        "40069cf306434810a9b5c306cc250b4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "67561c38a4db4ddd8b1bbf2e583efac0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5021c8f13a664c8e9a0876ccafd60dd1",
            "_dom_classes": [],
            "description": "models/electra/generator/config.json: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 463,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 463,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_844e8ec9146e49089aae027292c36278"
          }
        },
        "445908c8697442ee9c64839c34bfaeed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_59bce0ccdf7940948fff0868c2692f9d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 463/463 [00:01&lt;00:00, 405B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f0f9d32db62e4d059ab0d0684c5e438b"
          }
        },
        "5021c8f13a664c8e9a0876ccafd60dd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "844e8ec9146e49089aae027292c36278": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "59bce0ccdf7940948fff0868c2692f9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f0f9d32db62e4d059ab0d0684c5e438b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ad3c1b541a444cdc996048f1a6a52b29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1b875762584a4902a13d560edb9dd59e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ee27f00757bf4bf1b80def684374c0dd",
              "IPY_MODEL_2d0ad2d2d7104b5d8cd8b5b209bcc47a"
            ]
          }
        },
        "1b875762584a4902a13d560edb9dd59e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ee27f00757bf4bf1b80def684374c0dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c3a505dc395e432d8e97ab698fadc472",
            "_dom_classes": [],
            "description": "models/electra/discriminator/config.json: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 468,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 468,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3905087ed3244a489ab9a149326be06e"
          }
        },
        "2d0ad2d2d7104b5d8cd8b5b209bcc47a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2ac86e79a09048d39662c0dbd0a7b27c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 468/468 [00:12&lt;00:00, 38.5B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_43abb0e0a5a4453cb011f1dc7351fbb6"
          }
        },
        "c3a505dc395e432d8e97ab698fadc472": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3905087ed3244a489ab9a149326be06e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2ac86e79a09048d39662c0dbd0a7b27c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "43abb0e0a5a4453cb011f1dc7351fbb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "87fd34ef31ae4358a66bb496976445fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1bbc8375f71345e6b6534f40db55f325",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_332999fcd83140eaa0a072012c0333f0",
              "IPY_MODEL_3c5bbe8e5be74dd6be1e74d205926e3e"
            ]
          }
        },
        "1bbc8375f71345e6b6534f40db55f325": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "332999fcd83140eaa0a072012c0333f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c2eaf7bfadc345e898104189fa590709",
            "_dom_classes": [],
            "description": "models/electra/vocab.txt: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 81,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 81,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ad1cf7953ab442d8369c7101ebedc4b"
          }
        },
        "3c5bbe8e5be74dd6be1e74d205926e3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4e18c07f5e044e6bb15f7786d7fc0e0d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 81.0/81.0 [00:01&lt;00:00, 74.1B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9ae6d1034dac45d18519a182d79b64b1"
          }
        },
        "c2eaf7bfadc345e898104189fa590709": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ad1cf7953ab442d8369c7101ebedc4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e18c07f5e044e6bb15f7786d7fc0e0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9ae6d1034dac45d18519a182d79b64b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/agemagician/Prot-Transformers/blob/master/Embedding/Basic/Electra.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ua7KoVhV-mFs",
        "colab_type": "text"
      },
      "source": [
        "<h3> Extracting protein sequences' features using ProtElectra pretrained-model <h3>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9EYUbfGj-mFt",
        "colab_type": "text"
      },
      "source": [
        "<b>1. Load necessry libraries including huggingface transformers<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iNcbgAuH-xaw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "ee622b25-2ef0-4f1d-bb18-e58a5309638e"
      },
      "source": [
        "!pip install -q transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 675kB 8.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.8MB 24.6MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 62.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 55.7MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1x0wXUlL-mFv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from transformers import ElectraTokenizer, ElectraForPreTraining, ElectraForMaskedLM, ElectraModel, pipeline\n",
        "import re\n",
        "import numpy as np\n",
        "import os\n",
        "import requests\n",
        "from tqdm.auto import tqdm"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gIlqWxT9-mF2",
        "colab_type": "text"
      },
      "source": [
        "<b>2. Set the url location of ProtElectra and the vocabulary file<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UcsknDRC-mF3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generatorModelUrl = 'https://www.dropbox.com/s/5x5et5q84y3r01m/pytorch_model.bin?dl=1'\n",
        "discriminatorModelUrl = 'https://www.dropbox.com/s/9ptrgtc8ranf0pa/pytorch_model.bin?dl=1'\n",
        "\n",
        "generatorConfigUrl = 'https://www.dropbox.com/s/9059fvix18i6why/config.json?dl=1'\n",
        "discriminatorConfigUrl = 'https://www.dropbox.com/s/jq568evzexyla0p/config.json?dl=1'\n",
        "\n",
        "vocabUrl = 'https://www.dropbox.com/s/wck3w1q15bc53s0/vocab.txt?dl=1'"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GF_6od9_-mF8",
        "colab_type": "text"
      },
      "source": [
        "<b>3. Download ProtElectra models and vocabulary files<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mimKojYZ-mF9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "downloadFolderPath = 'models/electra/'"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zIqG1l0L-mGC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "discriminatorFolderPath = os.path.join(downloadFolderPath, 'discriminator')\n",
        "generatorFolderPath = os.path.join(downloadFolderPath, 'generator')\n",
        "\n",
        "discriminatorModelFilePath = os.path.join(discriminatorFolderPath, 'pytorch_model.bin')\n",
        "generatorModelFilePath = os.path.join(generatorFolderPath, 'pytorch_model.bin')\n",
        "\n",
        "discriminatorConfigFilePath = os.path.join(discriminatorFolderPath, 'config.json')\n",
        "generatorConfigFilePath = os.path.join(generatorFolderPath, 'config.json')\n",
        "\n",
        "vocabFilePath = os.path.join(downloadFolderPath, 'vocab.txt')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RQqswOx5-mGF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if not os.path.exists(discriminatorFolderPath):\n",
        "    os.makedirs(discriminatorFolderPath)\n",
        "if not os.path.exists(generatorFolderPath):\n",
        "    os.makedirs(generatorFolderPath)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rtQesVAJBP7e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def download_file(url, filename):\n",
        "  response = requests.get(url, stream=True)\n",
        "  with tqdm.wrapattr(open(filename, \"wb\"), \"write\", miniters=1,\n",
        "                    total=int(response.headers.get('content-length', 0)),\n",
        "                    desc=filename) as fout:\n",
        "      for chunk in response.iter_content(chunk_size=4096):\n",
        "          fout.write(chunk)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YDLH2U6S-mGJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262,
          "referenced_widgets": [
            "01d8ad27e05a4175b9454d7ebc2363ed",
            "4894baf418d441b89f9694c33c4a8221",
            "572c7ef1afd547e8a5dfe5ee26162200",
            "9d2fe16e49164d32b42823907c980af0",
            "76dec96200924d8393a421001a7a54bd",
            "f6bd2538ae1b42c9aed28b231d3cee9e",
            "6af790cfb9c446679fe60fd8a98078db",
            "0d24fa9e2c2e4619ad9e78bc7b054c7f",
            "16277ad20d5349a8a737a54acd2ee506",
            "803da87847ce429d9b06b3905ccf24be",
            "7277eafb8b5f47f39f1709be9ebfcce3",
            "2bd8bc9cf0c64c419781a87d97842bff",
            "927fae41ae594686b6504d1c4ec84924",
            "1c7a033d87f94f6eb5aceb5e49abe8c5",
            "712c6b0ebfed4b189d3415eee814ce2e",
            "c90762cb282b40eea7c3d084c8841b0e",
            "14e2f1e4f16845a2bf29b4bf3cdfc008",
            "40069cf306434810a9b5c306cc250b4d",
            "67561c38a4db4ddd8b1bbf2e583efac0",
            "445908c8697442ee9c64839c34bfaeed",
            "5021c8f13a664c8e9a0876ccafd60dd1",
            "844e8ec9146e49089aae027292c36278",
            "59bce0ccdf7940948fff0868c2692f9d",
            "f0f9d32db62e4d059ab0d0684c5e438b",
            "ad3c1b541a444cdc996048f1a6a52b29",
            "1b875762584a4902a13d560edb9dd59e",
            "ee27f00757bf4bf1b80def684374c0dd",
            "2d0ad2d2d7104b5d8cd8b5b209bcc47a",
            "c3a505dc395e432d8e97ab698fadc472",
            "3905087ed3244a489ab9a149326be06e",
            "2ac86e79a09048d39662c0dbd0a7b27c",
            "43abb0e0a5a4453cb011f1dc7351fbb6",
            "87fd34ef31ae4358a66bb496976445fa",
            "1bbc8375f71345e6b6534f40db55f325",
            "332999fcd83140eaa0a072012c0333f0",
            "3c5bbe8e5be74dd6be1e74d205926e3e",
            "c2eaf7bfadc345e898104189fa590709",
            "3ad1cf7953ab442d8369c7101ebedc4b",
            "4e18c07f5e044e6bb15f7786d7fc0e0d",
            "9ae6d1034dac45d18519a182d79b64b1"
          ]
        },
        "outputId": "fefb1ff7-3663-4bb2-85e2-bea0f5e4408c"
      },
      "source": [
        "if not os.path.exists(generatorModelFilePath):\n",
        "    download_file(generatorModelUrl, generatorModelFilePath)\n",
        "\n",
        "if not os.path.exists(discriminatorModelFilePath):\n",
        "    download_file(discriminatorModelUrl, discriminatorModelFilePath)\n",
        "    \n",
        "if not os.path.exists(generatorConfigFilePath):\n",
        "    download_file(generatorConfigUrl, generatorConfigFilePath)\n",
        "\n",
        "if not os.path.exists(discriminatorConfigFilePath):\n",
        "    download_file(discriminatorConfigUrl, discriminatorConfigFilePath)\n",
        "    \n",
        "if not os.path.exists(vocabFilePath):\n",
        "    download_file(vocabUrl, vocabFilePath)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "01d8ad27e05a4175b9454d7ebc2363ed",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/generator/pytorch_model.bin', max=26098224…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "16277ad20d5349a8a737a54acd2ee506",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/discriminator/pytorch_model.bin', max=1679…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "14e2f1e4f16845a2bf29b4bf3cdfc008",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/generator/config.json', max=463.0, style=P…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ad3c1b541a444cdc996048f1a6a52b29",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/discriminator/config.json', max=468.0, sty…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "87fd34ef31ae4358a66bb496976445fa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/electra/vocab.txt', max=81.0, style=ProgressStyle(…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lPn__UyU-mGM",
        "colab_type": "text"
      },
      "source": [
        "<b>4. Load the vocabulary and ProtElectra discriminator and generator Models<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MyDuw-5e-mGN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = ElectraTokenizer(vocabFilePath, do_lower_case=False )"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nd7iRBwA-mGQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "discriminator = ElectraForPreTraining.from_pretrained(discriminatorFolderPath)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uRMq3Y2x-mGY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generator = ElectraForMaskedLM.from_pretrained(generatorFolderPath)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXTS5XTn-mGc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "electra = ElectraModel.from_pretrained(discriminatorFolderPath)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EDSyqG-7-mGg",
        "colab_type": "text"
      },
      "source": [
        "<b>5. Load the models into the GPU if avilabile<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S1goaPZO-mGg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "discriminator = pipeline('feature-extraction', model=discriminator, tokenizer=tokenizer,device=0)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ef7Y8ARx-mGk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generator = pipeline('feature-extraction', model=generator, tokenizer=tokenizer,device=0)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6LvG0Kk0-mGo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "electra = pipeline('feature-extraction', model=electra, tokenizer=tokenizer,device=0)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dK4JQxE--mGr",
        "colab_type": "text"
      },
      "source": [
        "<b>6. Create or load sequences and map rarely occured amino acids (U,Z,O,B) to (X)<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h5XBh2qI-mGs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequences_Example = [\"A E T C Z A O\",\"S K T Z P\"]"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tIei38iw-mGu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequences_Example = [re.sub(r\"[UZOB]\", \"X\", sequence) for sequence in sequences_Example]"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O4foaphT-mGw",
        "colab_type": "text"
      },
      "source": [
        "<b>7. Extracting sequences' features and covert the output to numpy if needed<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KAdAUoTZ-mGw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "discriminator_embedding = discriminator(sequences_Example)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tqrT2mIo-mGy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generator_embedding = generator(sequences_Example)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MPmuXhLq-mG1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "electra_embedding = electra(sequences_Example)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CVpr-qEC-mG4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "discriminator_embedding = np.array(discriminator_embedding)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a7juAjOP-mHA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generator_embedding = np.array(generator_embedding)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBc_p9bu-mHD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "electra_embedding = np.array(electra_embedding)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FZDa17Je-mHH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "942941fc-45fd-4d91-c5a1-51d344dd39e1"
      },
      "source": [
        "print(discriminator_embedding)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[-10.04450417  -1.24294174  -1.15765846  -1.16310275  -1.01715338\n",
            "   -1.45999277  -1.46373045  -1.45527709 -11.15840149]\n",
            " [ -9.85941887  -1.03460002  -0.97092319  -1.22789395  -1.90789914\n",
            "   -1.42081475 -10.74413872  -0.8445996   -0.76843512]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d5nD3g7B-mHO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "1f27fb3a-6190-40ea-cc0f-47b2f4ce322f"
      },
      "source": [
        "print(generator_embedding)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[-1.23957577e+01 -1.27599640e+01 -3.73967285e+01 -2.04540882e+01\n",
            "   -1.84475422e+01  4.21890378e-01 -1.41020656e+00  1.16866791e+00\n",
            "    6.08514261e+00  1.03796988e+01 -8.59528351e+00  1.14875287e-01\n",
            "    4.69288778e+00  1.02395563e+01  3.29151654e+00 -5.13710678e-01\n",
            "   -4.13310766e+00 -6.46999168e+00  1.69129694e+00 -6.73366785e+00\n",
            "   -2.65442085e+00 -5.94073105e+00 -3.86548686e+00 -4.43838501e+00\n",
            "    7.09534502e+00  2.54479647e+00 -1.27812948e+01 -1.27303696e+01\n",
            "   -1.22453728e+01 -1.29130573e+01]\n",
            "  [-1.28436556e+01 -1.32133760e+01 -3.39164391e+01 -1.80228500e+01\n",
            "   -1.99971027e+01 -8.79043221e-01  2.14121113e+01  3.00555944e+00\n",
            "    6.51581669e+00  6.91750765e+00 -6.56447697e+00 -2.84736991e+00\n",
            "    4.84939623e+00  7.71299458e+00 -3.31786108e+00  2.78462839e+00\n",
            "    8.20150375e+00 -1.02165098e+01  2.70752811e+00 -2.60611963e+00\n",
            "   -1.54998708e+00 -3.94209218e+00 -5.67769909e+00 -8.36740017e+00\n",
            "   -7.96070397e-01  8.07460213e+00 -1.29483957e+01 -1.21770735e+01\n",
            "   -1.29219055e+01 -1.29152012e+01]\n",
            "  [-1.64518929e+01 -1.65454979e+01 -3.23558083e+01 -2.44598274e+01\n",
            "   -2.06531677e+01  1.01017058e+00  1.04712796e+00  4.86245006e-01\n",
            "    4.94201869e-01  1.84742987e+00  9.01876986e-01  7.32019961e-01\n",
            "    6.88322425e-01  6.87870562e-01  2.18310237e-01  2.51740724e-01\n",
            "    1.71351418e-01 -5.06338239e-01  4.49751288e-01  3.87506038e-01\n",
            "   -5.16622961e-02 -2.63717622e-01 -2.31056303e-01 -2.02970266e-01\n",
            "   -4.47634548e-01 -2.16323590e+00 -1.65033646e+01 -1.65888233e+01\n",
            "   -1.65016594e+01 -1.65399723e+01]\n",
            "  [-1.50611944e+01 -1.50300789e+01 -5.81444016e+01 -2.80720272e+01\n",
            "   -2.58537483e+01 -9.50347543e-01  2.74457526e+00  1.60952127e+00\n",
            "   -6.49901032e-01  5.06223142e-01  6.69786310e+00 -2.95269275e+00\n",
            "    3.48857927e+00  1.68974090e+00 -1.34841216e+00  2.34007339e+01\n",
            "    9.76615238e+00 -6.32839537e+00  9.77743721e+00 -7.32272243e+00\n",
            "   -7.79554081e+00 -2.66154313e+00 -2.93380857e-01 -3.55062008e+00\n",
            "   -5.42144537e+00  6.65978861e+00 -1.52309008e+01 -1.46715984e+01\n",
            "   -1.48442726e+01 -1.54420252e+01]\n",
            "  [-4.93959332e+00 -4.84602833e+00 -3.20939674e+01 -2.62943020e+01\n",
            "   -2.08584709e+01  3.38700438e+00 -9.50319386e+00  5.65627003e+00\n",
            "   -1.56183329e+01 -1.39901333e+01  1.38417339e+01 -1.52644091e+01\n",
            "   -5.12796021e+00 -4.90904903e+00 -2.13959843e-01 -4.25262642e+00\n",
            "   -1.89855349e+00  4.42132616e+00 -7.26454115e+00  1.71410599e+01\n",
            "   -3.17566657e+00 -1.85871470e+00  1.41687489e+01  2.46558399e+01\n",
            "   -1.78141189e+00  1.18934364e+01 -5.38892174e+00 -5.64560509e+00\n",
            "   -5.24885941e+00 -6.63561583e+00]\n",
            "  [-9.59953690e+00 -9.42864990e+00 -4.34309578e+01 -3.05225868e+01\n",
            "   -1.46501646e+01  1.39394772e+00 -1.88640344e+00 -2.33593035e+00\n",
            "   -1.20098829e+01 -7.57416058e+00  2.03404980e+01 -7.51384687e+00\n",
            "    2.06306505e+00 -3.31826836e-01 -4.22247171e+00  2.77796221e+00\n",
            "    2.18072414e+00  1.14093828e+01  1.21742601e+01  7.68305588e+00\n",
            "   -4.76336527e+00  1.80259953e+01  5.71142912e+00  6.13513851e+00\n",
            "   -6.04661131e+00  1.06718998e+01 -1.03325062e+01 -9.84378052e+00\n",
            "   -9.59133816e+00 -1.03003750e+01]\n",
            "  [-1.29778614e+01 -1.32217941e+01 -3.27495613e+01 -1.71088276e+01\n",
            "   -1.92356243e+01 -1.13175476e+00  1.91034451e+01 -2.34235096e+00\n",
            "    4.54130697e+00  6.36992788e+00 -3.76075029e+00 -8.24522436e-01\n",
            "    4.82306671e+00  4.21603584e+00 -2.88708782e+00  3.13465214e+00\n",
            "    6.96841526e+00 -7.48439837e+00  4.97207928e+00 -4.25587922e-01\n",
            "   -7.27726877e-01 -3.03649092e+00 -2.96945167e+00 -6.95996714e+00\n",
            "   -1.00688803e+00  7.05908537e+00 -1.29352446e+01 -1.25057907e+01\n",
            "   -1.31224728e+01 -1.28113804e+01]\n",
            "  [-4.57653332e+00 -4.20836544e+00 -5.22899399e+01 -2.03918571e+01\n",
            "   -2.07999649e+01  5.69703341e+00 -2.67108750e+00  1.79081810e+00\n",
            "   -1.29915152e+01 -1.01639271e+01  1.58602409e+01 -1.33040142e+01\n",
            "   -1.88877642e-01 -8.45543087e-01 -4.51154566e+00  1.03116760e+01\n",
            "    6.32175255e+00  9.04323995e-01  1.19482269e+01  2.79872084e+00\n",
            "   -9.13107681e+00  5.92987394e+00  1.12684107e+01  2.20545635e+01\n",
            "   -3.23027587e+00  2.52665672e+01 -5.55826950e+00 -5.22262192e+00\n",
            "   -4.70096016e+00 -6.70619583e+00]\n",
            "  [-2.45733929e+00 -1.95955944e+00 -1.59391193e+01 -1.85039291e+01\n",
            "   -5.89937019e+00 -6.50298786e+00 -9.43085957e+00 -8.68498230e+00\n",
            "   -1.31727972e+01 -1.02126303e+01 -9.00458395e-01  3.53697610e+00\n",
            "    8.37642193e+00  1.52279913e+00  1.60721827e+00 -1.04391489e+01\n",
            "   -1.45915723e+00  2.23125324e+01 -6.10481930e+00  1.78240013e+01\n",
            "    2.11072102e+01  5.28813171e+00  1.04757767e+01 -2.00674295e+00\n",
            "   -4.98155355e+00  1.15991554e+01 -2.47656250e+00 -3.96287918e-01\n",
            "   -1.95845032e+00 -2.51840782e+00]]\n",
            "\n",
            " [[-1.16956234e+01 -1.19816723e+01 -3.21071930e+01 -2.13521099e+01\n",
            "   -1.54733248e+01 -1.90980017e+00  5.85383368e+00  1.46598387e+00\n",
            "    1.23547506e+01  1.53836823e+01 -9.27897930e+00 -4.27176982e-01\n",
            "    6.30868959e+00  1.41463308e+01 -8.19639921e-01 -5.14878452e-01\n",
            "   -3.69310760e+00 -8.13374996e+00  5.13372779e-01 -4.98852730e+00\n",
            "   -1.66855216e+00 -5.92734528e+00 -6.74383307e+00 -5.84086704e+00\n",
            "    5.44813538e+00  4.36422682e+00 -1.19224472e+01 -1.18120432e+01\n",
            "   -1.14241085e+01 -1.18614359e+01]\n",
            "  [-9.68261909e+00 -9.71635437e+00 -4.13889008e+01 -3.08279819e+01\n",
            "   -1.85205994e+01 -3.81829214e+00 -9.77860153e-01  7.27158785e-01\n",
            "   -1.00444431e+01 -3.82601333e+00  1.97499218e+01 -1.09081535e+01\n",
            "    1.10642552e+00 -1.65712094e+00  6.04737043e-01  2.03259850e+00\n",
            "    4.25734663e+00  1.31386414e+01  5.56363916e+00  3.88423014e+00\n",
            "   -4.01852703e+00  9.17521095e+00  2.01495957e+00  4.18393564e+00\n",
            "   -5.47827625e+00  7.02693510e+00 -1.02421303e+01 -1.01867638e+01\n",
            "   -9.70044231e+00 -1.04596329e+01]\n",
            "  [-1.18016453e+01 -1.18001547e+01 -3.75331345e+01 -1.80220642e+01\n",
            "   -1.53212099e+01  2.57545471e-01 -1.78236830e+00 -2.74115300e+00\n",
            "   -3.87155175e+00 -1.61278319e+00 -3.26265478e+00  1.69696832e+00\n",
            "    1.62194023e+01  1.42366338e+00 -3.01669145e+00 -2.40850687e+00\n",
            "   -1.39865980e-01  7.02023554e+00  2.35099030e+00 -2.39906120e+00\n",
            "    6.66310215e+00  4.65158892e+00 -1.58046567e+00 -3.70895553e+00\n",
            "   -4.17098141e+00  4.93618441e+00 -1.18105936e+01 -1.07474327e+01\n",
            "   -1.17359152e+01 -1.20621023e+01]\n",
            "  [-1.45739012e+01 -1.45232973e+01 -4.83617096e+01 -1.80280151e+01\n",
            "   -2.09274998e+01  6.22182965e-01  1.29912300e+01 -9.52475548e-01\n",
            "    1.73681984e+01  1.37555084e+01 -8.64837646e+00 -8.89269292e-01\n",
            "    6.38695717e+00  1.03171825e+01 -4.35753137e-01  2.02519073e+01\n",
            "    3.51847839e+00 -1.18193598e+01  5.52459764e+00 -8.08372402e+00\n",
            "   -3.94129038e+00 -6.14824247e+00 -5.51175117e+00 -9.13353348e+00\n",
            "   -9.51043189e-01  9.22923088e+00 -1.45646744e+01 -1.40639267e+01\n",
            "   -1.40897322e+01 -1.46457167e+01]\n",
            "  [-6.25180149e+00 -6.14815998e+00 -4.01920242e+01 -5.16830635e+00\n",
            "   -2.16563168e+01  2.46153908e+01 -5.15479612e+00 -3.59241486e+00\n",
            "   -6.00791073e+00 -1.16094389e+01  2.18240690e+00  1.03010285e+00\n",
            "   -7.84206212e-01 -1.41077316e+00 -9.18387699e+00  2.84730411e+00\n",
            "    9.62571383e-01 -4.35119295e+00  6.52140665e+00  9.25898075e+00\n",
            "   -9.04497814e+00  1.78764229e+01  3.97166705e+00  1.56725349e+01\n",
            "    3.89012432e+00  1.82872715e+01 -7.50820494e+00 -7.10393047e+00\n",
            "   -6.45093822e+00 -8.36666679e+00]\n",
            "  [-1.71594734e+01 -1.72645264e+01 -3.83741226e+01 -2.51319637e+01\n",
            "   -2.26372032e+01 -1.73624098e-01  2.65587306e+00  3.80683565e+00\n",
            "   -4.69628572e+00 -7.48541117e-01  4.07231331e+00 -3.46936035e+00\n",
            "    7.91141450e-01  1.55040175e-01  5.08213788e-03  1.41644788e+00\n",
            "    1.17671013e+01 -1.85262287e+00  1.64762962e+00 -1.38723493e+00\n",
            "   -2.74782395e+00 -9.52501893e-01 -1.83195901e+00 -1.68984342e+00\n",
            "   -1.90868473e+00 -4.32874680e-01 -1.71976585e+01 -1.66955299e+01\n",
            "   -1.75605545e+01 -1.75066128e+01]\n",
            "  [-1.44474792e+00 -1.31949329e+00 -1.16342087e+01 -1.22167826e+01\n",
            "   -1.17139835e+01 -3.09200585e-01 -1.30206795e+01 -1.00540962e+01\n",
            "   -1.38018589e+01 -1.47046385e+01 -2.19959545e+00  7.22117758e+00\n",
            "    4.28948259e+00  7.12590158e-01 -1.83151275e-01 -1.29453382e+01\n",
            "   -2.87362027e+00  1.33107166e+01 -4.94436026e+00  2.67790260e+01\n",
            "    1.58610706e+01  9.90043831e+00  1.42858658e+01  2.12256241e+00\n",
            "    1.34191656e+00  7.84016752e+00 -1.87407207e+00 -4.25275803e-01\n",
            "   -1.38748169e+00 -2.32405281e+00]\n",
            "  [-4.63111973e+00 -5.03529072e+00 -6.18317795e+00 -1.17435551e+01\n",
            "   -1.17116604e+01  1.10814202e+00 -9.99245548e+00 -9.84281540e+00\n",
            "   -8.61722565e+00 -5.47761536e+00 -1.02031136e+01  1.74401608e+01\n",
            "    2.87642193e+00  2.00111532e+00 -3.75716001e-01 -1.29280453e+01\n",
            "   -3.17157555e+00 -8.05041909e-01 -3.74906349e+00  2.22391853e+01\n",
            "    1.79991055e+01  7.82317877e+00  1.23875809e+00 -4.69649363e+00\n",
            "    8.29632759e+00  2.83883333e+00 -4.84726143e+00 -3.90127182e+00\n",
            "   -4.51586151e+00 -4.82450676e+00]\n",
            "  [-7.60858536e+00 -8.25556183e+00 -2.07332001e+01 -1.02866325e+01\n",
            "   -1.49655399e+01  7.31258917e+00 -7.01929760e+00 -6.86795473e+00\n",
            "   -4.35631418e+00 -2.66848803e+00 -8.78604698e+00  1.69867458e+01\n",
            "    2.83761203e-01  7.88218081e-01 -3.22061348e+00 -8.72165775e+00\n",
            "   -8.76245379e-01 -6.47427320e+00 -6.41367555e-01  1.19647503e+01\n",
            "    7.13660908e+00  1.16706924e+01 -6.50102329e+00 -5.83656836e+00\n",
            "    9.77767754e+00  1.68795824e+00 -8.03357220e+00 -7.51647043e+00\n",
            "   -7.48708630e+00 -7.83776093e+00]]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xjV3Ooa4-mHP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        },
        "outputId": "04cd3bc7-39ab-45f0-d664-7e7e4fc9f8b5"
      },
      "source": [
        "print(electra_embedding)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[-1.04451440e-01  1.96046948e-01  7.24665597e-02 ...  4.70827371e-02\n",
            "   -1.38893381e-01 -1.83730334e-01]\n",
            "  [-3.11754458e-02 -1.18080616e-01 -1.51422679e-01 ... -8.80782455e-02\n",
            "   -2.03649044e-01  2.34545898e-02]\n",
            "  [-6.92143589e-02 -7.63380080e-02 -1.78088211e-02 ... -4.15132381e-02\n",
            "   -3.08615528e-02 -8.58288854e-02]\n",
            "  ...\n",
            "  [-5.05245999e-02 -9.02514085e-02  6.78477362e-02 ... -4.76730466e-02\n",
            "   -9.57428291e-02 -1.68221351e-02]\n",
            "  [ 3.07775717e-02  7.57525049e-05 -5.32222912e-02 ... -1.47995083e-02\n",
            "   -1.57044619e-01 -9.64660496e-02]\n",
            "  [ 2.91196234e-03 -3.36659327e-02  1.97648183e-02 ...  1.61298946e-01\n",
            "   -1.03283569e-01 -1.35708928e-01]]\n",
            "\n",
            " [[-2.07917616e-01  1.58023611e-01  4.76757959e-02 ...  6.73392266e-02\n",
            "   -1.69237971e-01 -1.67796656e-01]\n",
            "  [-6.04737513e-02 -1.60797983e-01 -1.63700715e-01 ... -7.67330825e-02\n",
            "   -1.51252389e-01 -4.52133343e-02]\n",
            "  [-9.30745900e-02 -5.02012298e-02 -1.62957162e-02 ... -2.65192648e-04\n",
            "   -2.70886812e-03 -2.37740427e-02]\n",
            "  ...\n",
            "  [-1.82417497e-01 -3.13445553e-02  9.74562578e-03 ...  1.72021762e-01\n",
            "   -3.92518342e-02 -6.70020133e-02]\n",
            "  [-1.15254454e-01 -1.04350224e-01 -6.74721599e-02 ... -1.60024539e-01\n",
            "   -1.68165743e-01 -9.01472792e-02]\n",
            "  [-1.34273499e-01 -8.18957239e-02 -5.45871817e-02 ... -1.61771551e-01\n",
            "   -2.02696547e-01 -9.44775492e-02]]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GydzkoTA-mHS",
        "colab_type": "text"
      },
      "source": [
        "<b>Optional: Remove padding ([PAD]) and special tokens ([CLS],[SEP]) that is added by ProtElectra model<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F17npln--mHT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "features = [] \n",
        "\n",
        "for seq_num in range(len(electra_embedding)):\n",
        "    seq_len = len(sequences_Example[seq_num].replace(\" \", \"\"))\n",
        "    start_Idx = 1\n",
        "    end_Idx = seq_len+1\n",
        "    seq_emd = electra_embedding[seq_num][start_Idx:end_Idx]\n",
        "    features.append(seq_emd)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AsystccF-mHU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "outputId": "98341f8d-1a6a-46aa-ee63-9911b5aa33c0"
      },
      "source": [
        "print(features)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[array([[-3.11754458e-02, -1.18080616e-01, -1.51422679e-01, ...,\n",
            "        -8.80782455e-02, -2.03649044e-01,  2.34545898e-02],\n",
            "       [-6.92143589e-02, -7.63380080e-02, -1.78088211e-02, ...,\n",
            "        -4.15132381e-02, -3.08615528e-02, -8.58288854e-02],\n",
            "       [ 3.80904488e-02, -1.71692267e-01, -5.64219430e-02, ...,\n",
            "        -1.18378937e-01, -9.77956504e-02,  2.44725216e-02],\n",
            "       ...,\n",
            "       [ 1.27263516e-01, -1.34989679e-01, -3.06518644e-01, ...,\n",
            "         3.99149172e-02, -4.54527065e-02, -3.57910693e-01],\n",
            "       [-5.05245999e-02, -9.02514085e-02,  6.78477362e-02, ...,\n",
            "        -4.76730466e-02, -9.57428291e-02, -1.68221351e-02],\n",
            "       [ 3.07775717e-02,  7.57525049e-05, -5.32222912e-02, ...,\n",
            "        -1.47995083e-02, -1.57044619e-01, -9.64660496e-02]]), array([[-6.04737513e-02, -1.60797983e-01, -1.63700715e-01, ...,\n",
            "        -7.67330825e-02, -1.51252389e-01, -4.52133343e-02],\n",
            "       [-9.30745900e-02, -5.02012298e-02, -1.62957162e-02, ...,\n",
            "        -2.65192648e-04, -2.70886812e-03, -2.37740427e-02],\n",
            "       [-4.23046909e-02, -1.51860267e-01, -6.50829077e-02, ...,\n",
            "        -1.45550948e-02, -7.37645999e-02, -5.66908680e-02],\n",
            "       [ 4.75764386e-02, -7.08769858e-02, -2.75032073e-01, ...,\n",
            "        -1.02747001e-01, -1.06809154e-01, -2.18676060e-01],\n",
            "       [-4.99716476e-02,  1.03257410e-02, -9.70151871e-02, ...,\n",
            "        -1.18601866e-01,  7.05851475e-03, -1.71629295e-01]])]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}